Resumen Resultados V2

En este notebook aplicamos los test estadísticos y el algoritmo decisión multicriterio a aquellos modelos que pensamos que sufren de menos overfitting.

import pandas as pd
from pandas import read_csv
import seaborn as sns

from sklearn.model_selection import TimeSeriesSplit
from sklearn.model_selection import train_test_split

import joblib as joblib

import matplotlib.pyplot as plt

import scikit_posthocs as scp
import scipy.stats as stats
import operator
import numpy as np
import statsmodels as st
import pingouin as pg

import math
import sklearn.metrics as sm

from plotly.subplots import make_subplots
import plotly.express as px
import plotly
import plotly.graph_objects as go
import flask

C:\Users\Carmen\anaconda3\lib\site-packages\outdated\utils.py:14: OutdatedPackageWarning: The package outdated is out of date. Your version is 0.2.0, the latest is 0.2.1.
Set the environment variable OUTDATED_IGNORE=1 to disable these warnings.
  return warn(

Eliminando modelos que padecen overfitting

df_predictor_lag3_escalado = read_csv('../Datos_preprocesados/predictor_lag3_escalado.csv', encoding='latin-1', sep = ',', na_values = ['NaN', 'NaT'])
df_predictor_lag3_escalado = df_predictor_lag3_escalado.set_index('Fecha')

df_predictor_lag5_escalado = read_csv('../Datos_preprocesados/predictor_lag5_escalado.csv', encoding='latin-1', sep = ',', na_values = ['NaN', 'NaT'])
df_predictor_lag5_escalado = df_predictor_lag5_escalado.set_index('Fecha')

X_train_3, X_test_3, y_train_3, y_test_3 = train_test_split(df_predictor_lag3_escalado.drop(['Incidentes'], axis=1),
                                        df_predictor_lag3_escalado['Incidentes'], train_size = 0.8, test_size = 0.2, 
                                        random_state = 42, shuffle = False)

X_train_5, X_test_5, y_train_5, y_test_5 = train_test_split(df_predictor_lag5_escalado.drop(['Incidentes'], axis=1),
                                        df_predictor_lag5_escalado['Incidentes'], train_size = 0.8, test_size = 0.2, 
                                        random_state = 42, shuffle = False)

Cargamos los modelos

modelos = []

modelos.append(joblib.load('rf_lag3_escalado.pkl'))
modelos.append(joblib.load('rf_lag5_escalado.pkl'))

modelos.append(joblib.load('regresion_lineal_lag3_escalado.pkl'))
modelos.append(joblib.load('regresion_lineal_lag5_escalado.pkl'))

modelos.append(joblib.load('lasso_lag3_escalado.pkl'))
modelos.append(joblib.load('lasso_lag5_escalado.pkl'))

modelos.append(joblib.load('svr_lag3_escalado.pkl'))
modelos.append(joblib.load('svr_lag5_escalado.pkl'))

modelos.append(joblib.load('gbr_lag3_escalado.pkl'))
modelos.append(joblib.load('gbr_lag5_escalado.pkl'))

modelos.append(joblib.load('en_lag3_escalado.pkl'))
modelos.append(joblib.load('en_lag5_escalado.pkl'))

nombres = ['RF Lag3', 'RF Lag5', 'LR Lag3', 'LR Lag5', 'Lasso Lag3', 'Lasso Lag5', 'SVR Lag3', 'SVR Lag5', 'GBR Lag3', 'GBR Lag5', 'EN Lag3', 'EN Lag5']

Función que predice el instante t tanto en train como en test para cada modelo y muestra las gráficas

def pred_train_test(modelo, lag):
    
    if lag == 3:
        X_train = X_train_3
        y_train = y_train_3
        X_test = X_test_3
        y_test = y_test_3
    if lag == 5:
        X_train = X_train_5
        y_train = y_train_5
        X_test = X_test_5
        y_test = y_test_5
        
    fig = make_subplots(rows=1, cols=2)
    
    # Predicción en train
    y_pred_train_modelo = modelo.predict(X_train)
    # Predicción en test
    y_pred_test_modelo = modelo.predict(X_test)
    
    fig.add_trace(go.Scatter(x = y_train.index, y = y_pred_train_modelo, mode = 'lines', name = 'Train Real'), row = 1, col = 1)
    fig.add_trace(go.Scatter(x = y_train.index, y = y_train, mode = 'lines', name = 'Pred Train'), row = 1, col = 1)
    
    fig.add_trace(go.Scatter(x = y_test.index, y = y_pred_test_modelo, mode = 'lines', name = 'Test Real'), row = 1, col = 2)
    fig.add_trace(go.Scatter(x = y_test.index, y = y_test, mode = 'lines', name = 'Pred test'), row = 1, col = 2)
    
    fig.update_layout(showlegend = True, title_text = 'Predicciones Train VS Test ' + str(nombres[i]))
    
    flask.Markup(fig)
    
    fig.show("notebook")
    
    mae_train = sm.mean_absolute_error(y_train, y_pred_train_modelo)
    mae_test = sm.mean_absolute_error(y_test, y_pred_test_modelo)
    
    rmse_train = math.sqrt(sm.mean_squared_error(y_train, y_pred_train_modelo))
    rmse_test = math.sqrt(sm.mean_squared_error(y_test, y_pred_test_modelo))
    
    return mae_train, mae_test, rmse_train, rmse_test

maes_train = []
maes_test = []
rmses_train = []
rmses_test = []

for i in range(0, len(modelos)):
    nombre = nombres[i]
    # Si es numero par
    if i % 2 == 0:
        lag = 3
    else:
        lag = 5
    mae_train, mae_test, rmse_train, rmse_test = pred_train_test(modelos[i], lag)
    maes_train.append(mae_train)
    maes_test.append(mae_test)
    rmses_train.append(rmse_train)
    rmses_test.append(rmse_test)

De forma gráfica parece que los modelos que más overfitting hacen son Ranfom Forest y GBR (tanto para lag3 como para lag5). Ahora cogemos las medidas de error en la predicción t tanto para train como para test de todos los modelos. Aquellos que tengan una diferencia > 0.2 serán los que consideremos que están sobre ajustando los datos

Leemos las tablas de errores de RMSE y MAE tanto para train como para test

RMSE_train = pd.read_csv('../Datos_preprocesados/RMSE_errores_train.csv')
RMSE_test = pd.read_csv('../Datos_preprocesados/RMSE_errores_test.csv')
RMSE_train.drop(columns = "Pliegues", inplace = True)
RMSE_test.drop(columns = "Predicciones", inplace = True)

MAE_train = pd.read_csv('../Datos_preprocesados/MAE_errores_train.csv')
MAE_test = pd.read_csv('../Datos_preprocesados/MAE_errores_test.csv')
MAE_train.drop(columns = "Pliegues", inplace  =True)
MAE_test.drop(columns = "Predicciones", inplace = True)

CC_train = pd.read_csv('../Datos_preprocesados/CC_train.csv')
CC_train.drop(columns = "Pliegues", inplace = True)
CC_test = pd.read_csv('../Datos_preprocesados/CC_test.csv')
CC_test.drop(columns = "Predicciones", inplace = True)

columnas = CC_train.columns

Calculamos las diferencias y aquellas que superen el valor de 0.2 serán los modelos que eliminaremos

maes_train = pd.Series(maes_train, index = columnas)
maes_test = pd.Series(maes_test, index = columnas)
rmses_train = pd.Series(rmses_train, index = columnas)
rmses_test = pd.Series(rmses_test, index = columnas)

RMSE_dif = round(rmses_test - rmses_train,2)
MAE_dif = round(maes_test - maes_train,2)

overfitting = []
for i in range(0, len(RMSE_dif)):
    if RMSE_dif[i] >= 0.2 or MAE_dif[i] >= 0.2:
        overfitting.append(i)

Eliminamos los modelos que han hecho overfitting

RMSE_train.drop(columns = RMSE_train.columns[overfitting], inplace=True)
RMSE_test.drop(columns = RMSE_test.columns[overfitting], inplace=True)
MAE_train.drop(columns = MAE_train.columns[overfitting], inplace=True)
MAE_test.drop(columns = MAE_test.columns[overfitting], inplace=True)
CC_train.drop(columns = CC_train.columns[overfitting], inplace=True)
CC_test.drop(columns = CC_test.columns[overfitting], inplace=True)

Tests estadísticos

Partimos de las tablas a las que hemos eliminado los modelos que sobre ajustan los datos y aplicamos los test estadísticos

def pairwise_test(df, parametric = True, method = 'bonferroni', decreasing = True, ties=True):
    # Función que devuelve la matriz de rankings.
    # El dataframe de entrada df debe tener los modelos en las columnas y 
    # las medidas en las filas. 
    # parametric = indica si se aplica un test paramétrico (ttest_rel) o no paramétrico (wilcoxon)
    # decreasing = indica el sentido en el que es mejor la medida utilizada. 
    # ties = indca si se incluyen los empates en el resultado final 
    
    if decreasing:
        relate = operator.lt
    else:
        relate = operator.gt
    
    if parametric:
        test = stats.ttest_rel
    else:
        test = stats.wilcoxon
        
    cols = df.columns
    pvalues = np.array([])
    significant = np.array([])
    ranks = pd.DataFrame({"wins":np.zeros(len(cols)),
                         "ties":np.zeros(len(cols)),
                         "losses":np.zeros(len(cols))})
    ranks.index= cols
    for index1 in range(len(cols)):
        for index2 in range(index1+1,len(cols)):
            T,p = test(df.iloc[:,index1],df.iloc[:,index2])
            pvalues = np.append(pvalues,p)
            
    significant,padjust,a1,a2 = st.stats.multitest.multipletests(pvalues,alpha = 0.05,method = method)

    
    pos = 0;
    for index1 in range(len(cols)):
        for index2 in range(index1+1,len(cols)):
            if not significant[pos]:
                ranks.iloc[index1,1] = ranks.iloc[index1,1]+1
                ranks.iloc[index2,1] = ranks.iloc[index2,1]+1
            else:  
                if relate(df.iloc[:,index1].mean(),df.iloc[:,index2].mean()):
                    ranks.iloc[index1,0] = ranks.iloc[index1,0]+1
                    ranks.iloc[index2,2] = ranks.iloc[index2,2]+1
                else:
                    ranks.iloc[index2,0] = ranks.iloc[index2,0]+1
                    ranks.iloc[index1,2] = ranks.iloc[index1,2]+1
            pos = pos + 1
                
    ranks["diff"] = ranks.wins-ranks.losses
    #ranks = ranks.sort_values(by="diff",ascending = False)
    if  not ties:
        ranks.drop('ties',axis = 'columns', inplace = True)
    return ranks.astype(int)

def tabla_wins_losses(criterio, w_l):
    #global tabla
    if criterio == 'CC':
        tabla = CC_train
        nemenyi = nemenyi_cc
    if criterio == 'MAE':
        tabla = MAE_train
        nemenyi = nemenyi_mae
    tam = len(tabla.columns)
    # Recorremos las columnas
    for j in range(tam):
        # Recorremos las filas evitando las ya leidas al ser la matriz simetrica
        for i in range(j+1,tam):
            # Si p-value < 0.05
            if nemenyi.iat[i, j] < 0.05:
                # Cogemos los modelos con diferencias significativas
                modelo1 = columnas[i]
                modelo2 = columnas[j]
                if criterio == 'CC':
                    # Tomamos su CC maximo entre todos los fold
                    cc_modelo1 = pd.Series(tabla[modelo1]).mean()
                    cc_modelo2 = pd.Series(tabla[modelo2]).mean()
                    # El modelo correspondiente al mayor CC se añade un Win y el otro un Loss
                    if cc_modelo1 > cc_modelo2:
                        # Añadimos un Win al modelo1
                        w_l.iat[i,0] = w_l.iat[i,0] + 1
                        # Añadimos un Loss al modelo2
                        w_l.iat[j,1] = w_l.iat[j,1] + 1
                    if cc_modelo1 < cc_modelo2:
                        # Añadimos un Win al modelo2
                        w_l.iat[j,0] = w_l.iat[j,0] + 1
                        # Añadimos un Loss al modelo1
                        w_l.iat[i,1] = w_l.iat[i,1] + 1     
                if criterio == 'MAE':
                    # Tomamos su MAE mínimo entre todos los fold
                    mae_modelo1 = pd.Series(tabla[modelo1]).mean()
                    mae_modelo2 = pd.Series(tabla[modelo2]).mean()
                    # El modelo correspondiente al mayor CC se añade un Win y el otro un Loss
                    if mae_modelo1 < mae_modelo2:
                        # Añadimos un Win al modelo1
                        w_l.iat[i,0] = w_l.iat[i,0] + 1
                        # Añadimos un Loss al modelo2
                        w_l.iat[j,1] = w_l.iat[j,1] + 1
                    if mae_modelo1 > mae_modelo2:
                        # Añadimos un Win al modelo2
                        w_l.iat[j,0] = w_l.iat[j,0] + 1
                        # Añadimos un Loss al modelo1
                        w_l.iat[i,1] = w_l.iat[i,1] + 1

RMSE

fig = px.box(RMSE_train, y = RMSE_train.columns)
flask.Markup(fig)
fig.show()

RMSE_train.apply(stats.shapiro)

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	EN_lag3	EN_lag5
0	0.926488	0.894526	0.881280	0.884258	0.874022	0.886963	0.880959	0.884850
1	0.414248	0.190580	0.134983	0.145985	0.111330	0.156691	0.133844	0.148267

Como ningún p-valor < 0.05, no podemos rechazar la hipótesis nula de que las distribuciones son normales. Aplicamos el test de Mauchly para comprobar la esfericidad

pg.sphericity(RMSE_train)

C:\Users\Carmen\anaconda3\lib\site-packages\pingouin\distribution.py:950: RuntimeWarning:

divide by zero encountered in double_scalars

SpherResults(spher=True, W=inf, chi2=-inf, dof=27, pval=1.0)

Como el p-valor > 0.05 aplicamos el test paramétrico de anova para ver si hay diferencias significativas entre los modelos. Nuestra hipotesis nula es H0: no existen diferencias significativas entre los modelos

RMSE_train.rm_anova()

C:\Users\Carmen\anaconda3\lib\site-packages\pingouin\distribution.py:950: RuntimeWarning:

divide by zero encountered in double_scalars

	Source	ddof1	ddof2	F	p-unc	np2	eps
0	Within	7	63	1.41345	0.215786	0.135733	0.299658

El p-valor es 0.21 > 0.05 luego aceptamos la hipotesis nula, no existen diferencias significativas

test = stats.friedmanchisquare(*[RMSE_train[col] for col in RMSE_train.columns])
test

FriedmanchisquareResult(statistic=16.366666666666674, pvalue=0.021969618535374515)

scp.posthoc_nemenyi_friedman(RMSE_train, melted=False)

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	EN_lag3	EN_lag5
LR_lag3	1.000000	0.900000	0.900000	0.751638	0.860792	0.900000	0.900000	0.860792
LR_lag5	0.900000	1.000000	0.642482	0.358005	0.900000	0.900000	0.587906	0.477509
Lasso_lag3	0.900000	0.642482	1.000000	0.900000	0.210653	0.860792	0.900000	0.900000
Lasso_lag5	0.751638	0.358005	0.900000	1.000000	0.068652	0.587906	0.900000	0.900000
SVR_lag3	0.860792	0.900000	0.210653	0.068652	1.000000	0.900000	0.172514	0.111334
SVR_lag5	0.900000	0.900000	0.860792	0.587906	0.900000	1.000000	0.806216	0.697062
EN_lag3	0.900000	0.587906	0.900000	0.900000	0.172514	0.806216	1.000000	0.900000
EN_lag5	0.860792	0.477509	0.900000	0.900000	0.111334	0.697062	0.900000	1.000000

No detecta diferencias significativas

rsme_ranks = pairwise_test(RMSE_train, parametric=False, decreasing=True, ties= False, method="holm")
rsme_ranks

	wins	losses	diff
LR_lag3	0	0	0
LR_lag5	0	0	0
Lasso_lag3	0	0	0
Lasso_lag5	0	0	0
SVR_lag3	0	0	0
SVR_lag5	0	0	0
EN_lag3	0	0	0
EN_lag5	0	0	0

MAE

fig = px.box(MAE_train, y = MAE_train.columns)
flask.Markup(fig)
fig.show()

MAE_train.apply(stats.shapiro)

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	EN_lag3	EN_lag5
0	0.935417	0.912743	0.886039	0.917546	0.869829	0.946054	0.894715	0.919158
1	0.503236	0.300369	0.152955	0.336945	0.099507	0.622112	0.191507	0.349987

En este caso los p-valores son todos mayores que el valor de significancia luego no podemos rechazar la hipotesis de que se siga una distribucion normal

pg.sphericity(MAE_train)

C:\Users\Carmen\anaconda3\lib\site-packages\pingouin\distribution.py:950: RuntimeWarning:

divide by zero encountered in double_scalars

SpherResults(spher=True, W=inf, chi2=-inf, dof=27, pval=1.0)

De nuevo no podemos rechazar la hipotesis nula

MAE_train.rm_anova()

C:\Users\Carmen\anaconda3\lib\site-packages\pingouin\distribution.py:950: RuntimeWarning:

divide by zero encountered in double_scalars

	Source	ddof1	ddof2	F	p-unc	np2	eps
0	Within	7	63	2.233976	0.04288	0.198859	0.367085

En este caso el p-valor si es menor que 0.05 luego rechazamos la hipotesis nula H0: no existen diferencias significativas. Aplicamos el test de Tukey para ver donde estan las diferencias

MAE_melted = MAE_train.melt()

MAE_melted["subject"] = [i for _ in range(8) for i in range(1, 11)]
MAE_melted.head(20)

	variable	value	subject
0	LR_lag3	0.032479	1
1	LR_lag3	0.004186	2
2	LR_lag3	0.001866	3
3	LR_lag3	0.003994	4
4	LR_lag3	0.011721	5
5	LR_lag3	0.016943	6
6	LR_lag3	0.026278	7
7	LR_lag3	0.036142	8
8	LR_lag3	0.052798	9
9	LR_lag3	0.020134	10
10	LR_lag5	0.014992	1
11	LR_lag5	0.003734	2
12	LR_lag5	0.003266	3
13	LR_lag5	0.002763	4
14	LR_lag5	0.007150	5
15	LR_lag5	0.016096	6
16	LR_lag5	0.026127	7
17	LR_lag5	0.032730	8
18	LR_lag5	0.042889	9
19	LR_lag5	0.024806	10

tukey_test = pg.pairwise_tukey(data = MAE_melted, dv = 'value', between = 'variable')
tukey_test

	A	B	mean(A)	mean(B)	diff	se	T	p-tukey	hedges
0	EN_lag3	EN_lag5	0.018395	0.016443	0.001952	0.006951	0.280893	0.9	0.120311
1	EN_lag3	LR_lag3	0.018395	0.020654	-0.002259	0.006951	-0.324974	0.9	-0.139192
2	EN_lag3	LR_lag5	0.018395	0.017455	0.000940	0.006951	0.135218	0.9	0.057916
3	EN_lag3	Lasso_lag3	0.018395	0.018104	0.000292	0.006951	0.041953	0.9	0.017969
4	EN_lag3	Lasso_lag5	0.018395	0.016971	0.001424	0.006951	0.204871	0.9	0.087750
5	EN_lag3	SVR_lag3	0.018395	0.022021	-0.003626	0.006951	-0.521668	0.9	-0.223439
6	EN_lag3	SVR_lag5	0.018395	0.019194	-0.000798	0.006951	-0.114849	0.9	-0.049192
7	EN_lag5	LR_lag3	0.016443	0.020654	-0.004211	0.006951	-0.605867	0.9	-0.259503
8	EN_lag5	LR_lag5	0.016443	0.017455	-0.001013	0.006951	-0.145675	0.9	-0.062395
9	EN_lag5	Lasso_lag3	0.016443	0.018104	-0.001661	0.006951	-0.238940	0.9	-0.102342
10	EN_lag5	Lasso_lag5	0.016443	0.016971	-0.000528	0.006951	-0.076022	0.9	-0.032562
11	EN_lag5	SVR_lag3	0.016443	0.022021	-0.005579	0.006951	-0.802560	0.9	-0.343750
12	EN_lag5	SVR_lag5	0.016443	0.019194	-0.002751	0.006951	-0.395742	0.9	-0.169503
13	LR_lag3	LR_lag5	0.020654	0.017455	0.003199	0.006951	0.460192	0.9	0.197108
14	LR_lag3	Lasso_lag3	0.020654	0.018104	0.002551	0.006951	0.366927	0.9	0.157161
15	LR_lag3	Lasso_lag5	0.020654	0.016971	0.003683	0.006951	0.529845	0.9	0.226942
16	LR_lag3	SVR_lag3	0.020654	0.022021	-0.001367	0.006951	-0.196694	0.9	-0.084247
17	LR_lag3	SVR_lag5	0.020654	0.019194	0.001461	0.006951	0.210125	0.9	0.090000
18	LR_lag5	Lasso_lag3	0.017455	0.018104	-0.000648	0.006951	-0.093264	0.9	-0.039947
19	LR_lag5	Lasso_lag5	0.017455	0.016971	0.000484	0.006951	0.069653	0.9	0.029834
20	LR_lag5	SVR_lag3	0.017455	0.022021	-0.004566	0.006951	-0.656885	0.9	-0.281355
21	LR_lag5	SVR_lag5	0.017455	0.019194	-0.001738	0.006951	-0.250067	0.9	-0.107108
22	Lasso_lag3	Lasso_lag5	0.018104	0.016971	0.001132	0.006951	0.162917	0.9	0.069780
23	Lasso_lag3	SVR_lag3	0.018104	0.022021	-0.003918	0.006951	-0.563621	0.9	-0.241409
24	Lasso_lag3	SVR_lag5	0.018104	0.019194	-0.001090	0.006951	-0.156802	0.9	-0.067161
25	Lasso_lag5	SVR_lag3	0.016971	0.022021	-0.005050	0.006951	-0.726538	0.9	-0.311189
26	Lasso_lag5	SVR_lag5	0.016971	0.019194	-0.002222	0.006951	-0.319720	0.9	-0.136941
27	SVR_lag3	SVR_lag5	0.022021	0.019194	0.002828	0.006951	0.406819	0.9	0.174247

test = stats.friedmanchisquare(*[MAE_train[col] for col in MAE_train.columns])
test

FriedmanchisquareResult(statistic=18.23333333333335, pvalue=0.010960450248375383)

nemenyi_mae = scp.posthoc_nemenyi_friedman(MAE_train, melted=False)
nemenyi_mae

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	EN_lag3	EN_lag5
LR_lag3	1.000000	0.900000	0.900000	0.900000	0.806216	0.900000	0.900000	0.254294
LR_lag5	0.900000	1.000000	0.900000	0.900000	0.172514	0.900000	0.900000	0.900000
Lasso_lag3	0.900000	0.900000	1.000000	0.900000	0.210653	0.900000	0.900000	0.860792
Lasso_lag5	0.900000	0.900000	0.900000	1.000000	0.172514	0.900000	0.900000	0.900000
SVR_lag3	0.806216	0.172514	0.210653	0.172514	1.000000	0.860792	0.172514	0.003165
SVR_lag5	0.900000	0.900000	0.900000	0.900000	0.860792	1.000000	0.900000	0.210653
EN_lag3	0.900000	0.900000	0.900000	0.900000	0.172514	0.900000	1.000000	0.900000
EN_lag5	0.254294	0.900000	0.860792	0.900000	0.003165	0.210653	0.900000	1.000000

Detecta diferencias en EN_Lag5 VS SVR_Lag3

mae_ranks = pairwise_test(MAE_train, parametric=False, decreasing=True, ties= False, method="holm")
mae_ranks

	wins	losses	diff
LR_lag3	0	0	0
LR_lag5	0	0	0
Lasso_lag3	0	0	0
Lasso_lag5	0	0	0
SVR_lag3	0	0	0
SVR_lag5	0	0	0
EN_lag3	0	0	0
EN_lag5	0	0	0

columnas = []

for c in nemenyi_mae.columns:
    columnas.append(c)

modelos = MAE_train.columns

ceros = [0]*len(modelos)

d = {'Wins': ceros, 'Losses': ceros}

wins_losses_mae = pd.DataFrame(data = d)
wins_losses_mae = wins_losses_mae.set_index(modelos)

tabla_wins_losses('MAE', wins_losses_mae)
wins_losses_mae

	Wins	Losses
LR_lag3	0	0
LR_lag5	0	0
Lasso_lag3	0	0
Lasso_lag5	0	0
SVR_lag3	0	1
SVR_lag5	0	0
EN_lag3	0	0
EN_lag5	1	0

CC

fig = px.box(CC_train, y = CC_train.columns)
flask.Markup(fig)
fig.show()

CC_train.apply(stats.shapiro)

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	EN_lag3	EN_lag5
0	0.958121	0.887266	0.957226	0.912596	0.934377	0.846537	0.948635	0.923018
1	0.764250	0.157936	0.753839	0.299302	0.492296	0.052824	0.652372	0.382816

De nuevo no podemos aceptar que los datos siguen una normal, aplicamos esferididad

pg.sphericity(CC_train)

SpherResults(spher=True, W=54513315152787.92, chi2=-204.0853646136714, dof=27, pval=1.0)

CC_train.rm_anova()

	Source	ddof1	ddof2	F	p-unc	np2	eps
0	Within	7	63	1.278961	0.27524	0.124425	0.248523

En este caso no podemos rechazar la hipotesis nula: no existen diferencias significativas entre los modelos

test = stats.friedmanchisquare(*[CC_train[col] for col in CC_train.columns])
test

FriedmanchisquareResult(statistic=12.65868263473052, pvalue=0.08087428551735754)

nemenyi_cc = scp.posthoc_nemenyi_friedman(CC_train, melted=False)
nemenyi_cc

	LR_lag3	LR_lag5	Lasso_lag3	Lasso_lag5	SVR_lag3	SVR_lag5	EN_lag3	EN_lag5
LR_lag3	1.000000	0.697062	0.900000	0.447819	0.900000	0.387524	0.900000	0.697062
LR_lag5	0.697062	1.000000	0.615194	0.900000	0.900000	0.900000	0.778928	0.900000
Lasso_lag3	0.900000	0.615194	1.000000	0.358005	0.833502	0.303467	0.900000	0.615194
Lasso_lag5	0.447819	0.900000	0.358005	1.000000	0.900000	0.900000	0.533327	0.900000
SVR_lag3	0.900000	0.900000	0.833502	0.900000	1.000000	0.900000	0.900000	0.900000
SVR_lag5	0.387524	0.900000	0.303467	0.900000	0.900000	1.000000	0.477509	0.900000
EN_lag3	0.900000	0.778928	0.900000	0.533327	0.900000	0.477509	1.000000	0.778928
EN_lag5	0.697062	0.900000	0.615194	0.900000	0.900000	0.900000	0.778928	1.000000

No detecta diferencias significativas

cc_ranks = pairwise_test(CC_train, parametric=False, decreasing=False, ties= False, method="holm")
cc_ranks

C:\Users\Carmen\anaconda3\lib\site-packages\scipy\stats\morestats.py:2957: UserWarning:

Exact p-value calculation does not work if there are ties. Switching to normal approximation.

C:\Users\Carmen\anaconda3\lib\site-packages\scipy\stats\morestats.py:2971: UserWarning:

Sample size too small for normal approximation.

	wins	losses	diff
LR_lag3	0	0	0
LR_lag5	0	0	0
Lasso_lag3	0	0	0
Lasso_lag5	0	0	0
SVR_lag3	0	0	0
SVR_lag5	0	0	0
EN_lag3	0	0	0
EN_lag5	0	0	0

columnas = []

for c in nemenyi_cc.columns:
    columnas.append(c)

modelos = CC_train.columns

ceros = [0]*len(modelos)

d = {'Wins': ceros, 'Losses': ceros}

wins_losses_cc = pd.DataFrame(data = d)
wins_losses_cc = wins_losses_cc.set_index(modelos)

tabla_wins_losses('CC', wins_losses_cc)
wins_losses_cc

	Wins	Losses
LR_lag3	0	0
LR_lag5	0	0
Lasso_lag3	0	0
Lasso_lag5	0	0
SVR_lag3	0	0
SVR_lag5	0	0
EN_lag3	0	0
EN_lag5	0	0

Generamos la tabla completa y ordenamos

wins_losses = wins_losses_cc + wins_losses_mae
wins_losses['Diff'] = wins_losses['Wins'] - wins_losses['Losses']
wins_losses

	Wins	Losses	Diff
LR_lag3	0	0	0
LR_lag5	0	0	0
Lasso_lag3	0	0	0
Lasso_lag5	0	0	0
SVR_lag3	0	1	-1
SVR_lag5	0	0	0
EN_lag3	0	0	0
EN_lag5	1	0	1

wins_losses_rank = wins_losses.sort_values('Diff', ascending = False)
wins_losses_rank

	Wins	Losses	Diff
EN_lag5	1	0	1
LR_lag3	0	0	0
LR_lag5	0	0	0
Lasso_lag3	0	0	0
Lasso_lag5	0	0	0
SVR_lag5	0	0	0
EN_lag3	0	0	0
SVR_lag3	0	1	-1